package com.chance.cc.crawler.development.scripts.autohome.carSeriesForum;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class AutoHomeCarSeriesForumCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AutoHomeCarSeriesForumCrawlerScript.class);//日志
    private static final String DOMAIN = "autohome";
    private static final String SITE = "carSeriesForum";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //   https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js\\?20131010";//动态链接
    //    https://www.autohome.com.cn/4658
    private static final String CarSeriesRegulars = "https://www.autohome.com.cn/\\d*";//车系链接


    //    https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list?page_num=1&page_size=50&club_bbs_type=c&club_bbs_id=3248&club_order_type=1
    //    https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list?page_num=1&page_size=50&club_bbs_type=c&club_bbs_id=4658&club_order_type=2
    private static final String ForumRegulars = "https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list\\?page_num=\\d*&page_size=50&club_bbs_type=c&club_bbs_id=\\d*&club_order_type=1";//论坛链接回复
    private static final String ForumTwoRegulars = "https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list\\?page_num=\\d*&page_size=50&club_bbs_type=c&club_bbs_id=\\d*&club_order_type=2";//论坛链接发布
    //    http://club.autohome.com.cn/bbs/thread/4af129c6e1805435/98204958-1.html
    private static final String ContentRegulars = "http://club.autohome.com.cn/bbs/thread/\\S*/\\S*.html";//帖子链接

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
        addUrlRegular(ForumRegulars);
        addUrlRegular(ForumTwoRegulars);
        addUrlRegular(ContentRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析详情和在售配置
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ForumRegulars)||crawlerRecord.getHttpRequest().getUrl().matches(ForumTwoRegulars)) {
            forumRequest(crawlerRecord, page, parsedLinks);
        }
        return parsedLinks;
    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "https://www.autohome.com.cn/" + name.get(i);//得到车系页面
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .needWashed(false)
                    .build();
            parsedLinks.add(record);
        }
    }

    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        crawlerRecord.tagsCreator().bizTags().addCustomKV("keys", "");
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String[] split = carSeriesUrl.split("/");
        String carSeriesId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
        String model = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Tag_Site_Info);
        String forumUrl = null;
        if (model.matches("回复")) {
            forumUrl = "https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list?page_num=1&page_size=50&club_bbs_type=c&club_bbs_id=" + carSeriesId + "&club_order_type=1";
        } else {
            forumUrl = "https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list?page_num=1&page_size=50&club_bbs_type=c&club_bbs_id=" + carSeriesId + "&club_order_type=2";
        }
        if (StringUtils.isBlank(forumUrl)) {
            return;
        }
        String carSeriesName = page.getHtml().xpath("//h1/a").get();//车系名
        String brandName = page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/text()").get();//品牌名
        if (carSeriesName==null){
            carSeriesName=page.getHtml().xpath("//div[@class=\"path\"]/a[3]/text()").get();
            String brandNameStr = page.getHtml().xpath("//div[@class=\"subnav-title-name\"]/a").get();//品牌名
            String[] split1 = brandNameStr.split("-");
            brandName=split1[0];
        }

        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        seriesMap.put(AutoVMCommonField.Field_Series_id, split[split.length - 1]);//车系id
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌


        CrawlerRequestRecord forumRecord = CrawlerRequestRecord.builder()//解析论坛
                .itemPageRequest(crawlerRecord)
                .recordKey(forumUrl)
                .httpUrl(forumUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .needWashed(true)
                .build();
        forumRecord.tagsCreator().bizTags().addCustomKV("carSeriesId", carSeriesId);
        parsedLinks.add(forumRecord);
    }


    private void forumRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String carSeriesId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("carSeriesId");
        String url = page.getRequest().getUrl();
        String[] split = url.replaceAll("&page_size=50", "/").replaceAll("page_num=", "/").replaceAll("club_order_type=", "/").split("/");
        String nextPageUrl = "https://club.autohome.com.cn/frontapi/data/page/club_get_topics_list?page_num=" + (Integer.parseInt(split[split.length - 3]) + 1) + "&page_size=50&club_bbs_type=c&club_bbs_id=" + carSeriesId + "&club_order_type=" + split[split.length - 1];
        CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .recordKey(nextPageUrl)
                .httpUrl(nextPageUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .build();
        parsedLinks.add(turnPageRequest);


        List<Selectable> nodes = page.getJson().jsonPath($_type + "..items[*]").nodes();//所有文章
        for (Selectable node : nodes) {
            try {
                String articleUrl = new Json(node.get()).jsonPath(".pc_url").get();//文章url

                String releaseTime = null;
                if (split[split.length-1]=="1"){
                    releaseTime = new Json(node.get()).jsonPath(".club_topic_lastPostDate").get();//时间
                }else {
                    releaseTime = new Json(node.get()).jsonPath(".publish_time").get();//时间
                }
                if (StringUtils.isBlank(releaseTime)){
                    continue;
                }
                long time = DateUtils.parseDate(releaseTime, "yyyy/MM/dd HH:mm:ss").getTime();//时间字符串
                CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析论坛
                        .itemPageRequest(crawlerRecord)
                        .recordKey(articleUrl)
                        .httpUrl(articleUrl)
                        .releaseTime(time)
                        .copyBizTags()
                        .copyResultTags()
                        .needWashed(true)
                        .resultLabelTag(article)
                        .build();
                articleRecord.tagsCreator().bizTags().addCustomKV("time", time + "");
                parsedLinks.add(articleRecord);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(saleWash(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }


    public List<CrawlerData> saleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String articleUrl = page.getRequest().getUrl();//文章url
            String releaseTime = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("time");//时间
            String[] articleId = articleUrl.split("/");//url的关键字
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), articleId[articleId.length - 2]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(articleUrl)
                    .releaseTime(Long.parseLong(releaseTime))
                    .addContentKV("releaseTimeToLong", releaseTime)
                    .addContentKV("articleUrl", articleUrl)
                    .addContentKV(AICCommonField.Field_Author, "论坛")
                    .flowInPipelineTag("redis")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 20) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
